# -*- coding: utf-8 -*-
# @Time    : 2025-08-08 16:18:59
# @Author  : zhangbh
# @Email   : 526059967@qq.com
# @File    : git_prod_scan.py
# @Software: PyCharm
import json
import os
from multiprocessing import Pool
from urllib.parse import quote

import gitlab
import requests
from lxml import etree

# 需要检索的内容关键字
search_contents = [
    '',
]
# 个人中心自己分配申请的key
token = ''
gitlab_url = ''
gl = gitlab.Gitlab(gitlab_url, token)
# 登录完成后，gitlab的cookie
private_Cookie = ''

headers = {
    'Cookie': private_Cookie
}


def get_all_projects():
    data = []
    for g in gl.groups.list(all=True):
        for p in g.projects.list(all=True):
            project = gl.projects.get(p.id)
            item = {
                "id": p.id,
                "group": g.name,
                "project": p.name
            }
            data.append(item)

    # 将数据保存为 JSON 文件
    with open('all_output.json', 'w') as file:
        json.dump(data, file, indent=4)


def getProject(projectId, keyword):
    """
    根据项目 ID 获取项目信息并打印项目链接
    """
    url = "%s/search?utf8=&snippets=&scope=&search=%s&project_id=%s" % (gitlab_url, keyword, projectId)
    response = requests.get(url, headers=headers)
    pagehtml = etree.HTML(response.text)
    try:
        li_element = pagehtml.xpath('//li[@data-qa-selector="code_tab"]')[0]
        projecttitle = li_element.xpath('.//span/text()')[0]

        if int(projecttitle) > 0:
            # 查询项目信息
            project = gl.projects.get(projectId)

            print("项目存在关键词(" + keyword + "),项目【" + project.path + "】" + url)
        else:
            return
    except IndexError:
        # 查询项目信息
        project = gl.projects.get(projectId)

        # 获取项目链接
        project_url = project.web_url
        print("项目请求异常：" + url)
        pass


def start_project():
    pool = Pool(processes=5)
    # 读取项目数据
    with open('all_output.json', 'r') as file:
        data = file.read()
    data = json.loads(data)

    # 并发调用 getProject 函数
    for keyword in search_contents:
        # print('等待查询(' + keyword + ')结果====')
        for i in data:
            id = i.get('id')
            # print('扫描项目:'+i.get('group')+i.get('project'))
            # pool.apply(func=getProject, args=(id, quote(keyword)))
            # 这种异步的比较快，但是项目中如果有做调用次数限制，就容易出异常
            pool.apply_async(func=getProject, args=(id, quote(keyword)))
        print('并发调用完毕,等待查询【' + keyword + '】结果====')
        # pool.apply(func=getProject, args=(id,keyword))
        # print('(' + keyword + ')扫描完毕====')
        print("================================================================")

    pool.close()
    pool.join()
    print("================================================================")
    print("所有项目已扫描完毕")


if __name__ == '__main__':
    file_path = 'all_output.json'
    if os.path.isfile(file_path):
        start_project()
    else:
        get_all_projects()
        start_project()
